import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Vector;

import java.sql.Connection;

import com.aliasi.matrix.SvdMatrix;

//@Sangeeta
//1. Preprocess
//2. remove stop words
//3. stemming
//4. Unique terms
//5. term incident matrix
//6. 
public class Big_Lsi_model
{
	 private double[][] TERM_DOCUMENT_MATRIX;
		

	    private  String[] TERMS ; //String [] conversion Of UTerms(vector)
	    private String [] DOCS;//consistin document array
	    private String args[];//consisting of query terms
	   
	    java.sql.Connection conn = null;
	    String database = "lsi";
	    String table = "chromedb_simple";
	    String lsi_score_table = "lsi_term_score";
	    String username = "root";
	    String paswd = "123";
	    int startIssue  = 0;
	    int endIssue = 100;
        Vector<Vector<String>> data = new Vector<Vector<String>>();
        Vector<String> UTerms = new Vector<String>();//to hold unique Terms
        int no_of_terms ;
        int no_of_documents;
        
        //LSI Parameters
        static final int NUM_FACTORS = 2;
        static double featureInit = 0.01;
       // static double initialLearningRate = 0.005;// Original
        static double initialLearningRate = 0.0005;
        static int annealingRate = 1000;
        static double regularization = 0.00;
        static double minImprovement = 0.0000;
        static int minEpochs = 10;
        static int maxEpochs = 50000;

	    //Method to read from data base
	    private void ReadData(Connection conn) 
	    {
	    	String str = "select issueid, title, description,comments from "+table +" limit "+startIssue+
	    	              ","+endIssue;
	    	System.out.println("Str="+str);
	    	preprocess pobj = new preprocess();
	    	try {
				   Statement stmt = conn.createStatement();
				   stmt.execute(str);
				   ResultSet rs= stmt.getResultSet();
				   
				   
				   while(rs.next())
				   {
					   Vector <String> temp =  new Vector<String>();
					   String issueid=rs.getString("issueid").toLowerCase().trim();
					   String title =  rs.getString("title").toLowerCase().trim();
					   String description = rs.getString("description").toLowerCase().trim();
					   String comments = rs.getString("comments").toLowerCase().trim();
					   
					   //System.out.println("TT"+title);
					   //System.out.println("dd"+description);
					   //System.out.println("cc"+comments);
					   
					   title = pobj.remove_SW_and_STEM(title.trim());
					   description = pobj.remove_SW_and_STEM(description.trim());
					   comments=pobj.remove_SW_and_STEM(comments.trim());
					   //System.out.println("TT"+title);
					   //System.out.println("dd"+description);
					   //System.out.println("cc"+comments);
					   
					   temp.add(issueid.trim());
					   temp.add(title.trim());
					   temp.add(description.trim());
					   temp.add(comments.trim());
					   data.add(temp);   
					   
				   }
				   if(rs!=null)
					   rs.close();
				   if(stmt!=null)
					   stmt.close();
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
	    	 
	    	// TODO Auto-generated method stub
			/*System.out.println("data="+data.size());
			for(int i=0;i<data.size();i++)
			{
				System.out.println("issue="+data.get(i).get(0));
				System.out.println("Title="+data.get(i).get(1));
				System.out.println("description="+data.get(i).get(2));
				System.out.println("comments="+data.get(i).get(3));
			}*/
	    	
	 }

private void getUniqTerms()
	    {
			for(int i=0;i<data.size();i++)
			{
				//String  str = data.get(i).get(0);//issueid
				String str  = data.get(i).get(1);//title
				str=   str+" "+data.get(i).get(2);//desc
				str =  str+" "+data.get(i).get(3);//comments
				//str = str.trim();
				String []str2 = str.split(" ");
				for(int j=0;j<str2.length;j++)
					if((!UTerms.contains(str2[j]))&&(str2[j]!=" ")&&(str2[j]!="\n")&&(str2[j]!="\t"))
					{
						if(str2[j]!="")
						{UTerms.add(str2[j]);
						  //System.out.println("str2[j]="+str2[j]);
						}
					}
						
				UTerms.remove("");
			}
		}


	    private void calculateTDM()
	    {
			//using data form readData()
	    	
	    	no_of_documents = data.size();
	    	no_of_terms =UTerms.size();
	    	
	    	System.out.println("Number of Terms="+no_of_terms);
	    	System.out.println("no_of_documenst="+no_of_documents);
	    	
	    	TERM_DOCUMENT_MATRIX = new double[no_of_terms][no_of_documents];
	    	for(int i=0;i<data.size();i++)
	    	{
	    		
	    		for(int j=0;j<UTerms.size();j++)
	    		{
	    			boolean flag = false;
	    		  	String tempTerm = UTerms.get(j);
	    		  	/*if((data.get(i).get(1).contains(tempTerm))||(data.get(i).get(2).contains(tempTerm))||(data.get(i).get(3).contains(tempTerm)))
	    		  	{
	    		  		TERM_DOCUMENT_MATRIX[j][i] = 1;
	    		  	}
	    		  	else
	    		  	{
	    		  		TERM_DOCUMENT_MATRIX[j][i] = 0;
	    		  	}*/
	    		  	TERM_DOCUMENT_MATRIX[j][i] = get_occurances(tempTerm,data.get(i));
	    		  	//System.out.println("I am in loop");
	    		  	
	    		}//for
	    	}//for
			
		}



	    //counts the occurances for term document matrix
	private double get_occurances(String tempTerm, Vector<String> document)
	{
		
		int occ = 0;
		int pos=0;
		String str = document.get(1) +" "+document.get(2)+" "+document.get(3);
		
		//System.out.println("Term="+tempTerm);
		//System.out.println("Str="+str);
		//System.out.println("loop2" );
		if(!tempTerm.isEmpty())
			while(str.indexOf(tempTerm,pos)!=-1)
				{ 
					//System.out.println("Post="+pos);
					//System.out.println("Term="+tempTerm);

					occ++;
					pos =str.indexOf(tempTerm, pos);
					pos++;		
					//System.out.println("loop1 " );
					//interupt();
				}
		
		
	
			return occ;
	}

	public void interupt()
	{
		
		BufferedReader br =  new BufferedReader(new InputStreamReader(System.in));
		System.out.println("Intr");
		try {
			br.readLine();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	

	private void print_various_values(SvdMatrix matrix, String args[]) 
	{
		double[] scales = matrix.singularValues();
		double [][] termVectors = matrix.leftSingularVectors();
		double[][] docVectors = matrix.rightSingularVectors();
		
		for(int i=0;i<scales.length;i++)
		{
			System.out.println("scales["+i+"]="+scales[i]);
		}
		interupt();
		System.out.println("\nSCALES");
		for (int k = 0; k < NUM_FACTORS; ++k)
		    System.out.printf("%d  %4.2f\n",k,scales[k]);
		

		System.out.println("\nTERM VECTORS");
		for (int i = 0; i < termVectors.length; ++i) {
		    System.out.print("(");
		    for (int k = 0; k < NUM_FACTORS; ++k) {
			if (k > 0) System.out.print(", ");
			//System.out.printf("% 5.2f",termVectors[i][k]);
			System.out.println(termVectors[i][k]);
		    }
		    System.out.print(")  ");
		    System.out.println(TERMS[i]);
		}
		
		System.out.println("\nDOC VECTORS");
		for (int j = 0; j < docVectors.length; ++j) {
		    System.out.print("(");
		    for (int k = 0; k < NUM_FACTORS; ++k) {
			if (k > 0) System.out.print(", ");
			System.out.println(docVectors[j][k]);
		    }
		    System.out.print(")  ");
		    System.out.println(DOCS[j]);
		}
		
		
		//Initilize queries vector run Time		
		
		for (int m = 0; m < args.length; ++m) 
		{
		    search(scales,termVectors,docVectors,args[m]);
		}
			
	}

private void search(double[] scales, double[][] termVectors, double[][] docVectors,
             String arg)
	{
		String[] terms = arg.split(" |,"); // space or comma separated

		double[] queryVector = new double[NUM_FACTORS];
		Arrays.fill(queryVector,0.0);

		for (String term : terms)
			addTermVector(term,termVectors,queryVector);


			System.out.println("\nQuery=" + Arrays.asList(terms));
			System.out.print("Query Vector=(");
			for (int k = 0; k < queryVector.length; ++k)
			{
				if (k > 0) System.out.print(", ");
					System.out.printf("% 5.2f",queryVector[k]);
			}
			System.out.println(" )");

			System.out.println("\nDOCUMENT SCORES VS. QUERY");
			for (int j = 0; j < docVectors.length; ++j) 
			{
				//double score = dotProduct(queryVector,docVectors[j],scales);
				 double score = cosine(queryVector,docVectors[j],scales);
				System.out.printf("  %d: % 5.2f  %s\n",j,score,DOCS[j]);
			}

			System.out.println("\nTERM SCORES VS. QUERY");
			for (int i = 0; i < termVectors.length; ++i) 
			{
				//double score = dotProduct(queryVector,termVectors[i],scales);
				double score = cosine(queryVector,termVectors[i],scales);
				System.out.printf("  %d: % 5.2f  %s\n",i,score,TERMS[i]);
				write_to_sql(arg, TERMS[i],score,conn);
			}	
	}

//writing scores to database table
private void write_to_sql(String query, String term, double score, Connection conn2)
{
	if(Double.isNaN(score))
	{
		score =-10000.0;
	}
	String str = "insert into "+lsi_score_table+ " values( '"+query+"','"+term+"',"+score+")";
	if(conn!=null)
	{
		try 
		{
			Statement stmt = conn.createStatement();
			stmt.executeUpdate(str);
			if(stmt!=null)
				stmt.close();
			
		
		} catch (SQLException e) 
		{ e.printStackTrace();
		}
		
	}
	
}


private void addTermVector(String term, double[][] termVectors, double[] queryVector)
{

	
	for (int i = 0; i < TERMS.length; ++i) 
        {
		 //System.out.println("Helo");
        if (TERMS[i].equals(term)) 
          {
	        for (int j = 0; j < NUM_FACTORS; ++j) 
	          {
	            queryVector[j] += termVectors[i][j];
    	      }
         }//if
}//for
//for
	return;
    
}


private double dotProduct(double[] xs, double[] ys, double[] scales) 
{
	double sum = 0.0;
	for (int k = 0; k < xs.length; ++k)
      sum += xs[k] * ys[k] * scales[k];
 
	return sum;
}

 double cosine(double[] xs, double[] ys, double[] scales) 
{
	double product = 0.0;
	double xsLengthSquared = 0.0;
	double ysLengthSquared = 0.0;
	for (int k = 0; k < xs.length; ++k) 
	{
		double sqrtScale = Math.sqrt(scales[k]);
		double scaledXs = sqrtScale * xs[k];
		double scaledYs = sqrtScale * ys[k];
		xsLengthSquared += scaledXs * scaledXs;
		ysLengthSquared += scaledYs * scaledYs;
		product += scaledXs * scaledYs;
    }
return product / Math.sqrt(xsLengthSquared * ysLengthSquared);
}
	
//Method to create docs
	private  String[] create_docs() 
	{
		
		DOCS =  new String[data.size()];
		for(int i=0;i<data.size();i++)
		{
			String temp = data.get(i).get(1)+" "; //title
			temp =temp+data.get(i).get(2)+" ";//description
			temp = temp+data.get(i).get(3);//comment
			DOCS[i] = temp;
		}
		
		return DOCS;
		
	}


	/*private void remove_stop_words()
	{
		preprocess pobj =  new preprocess();
		for(int i=0; i<data.size();i++)
		{
			String title_str =  pobj.remove_stop_words(data.get(i).get(1));
			String description_str = pobj.remove_stop_words(data.get(i).get(2));
			String comments_str = pobj.remove_stop_words(data.get(i).get(3));
			
			Vector<String>
			data.get(i).set(1, )
			
		}	
		
	}*/	
public static void main(String args[])
     {
	  Big_Lsi_model blsim  =  new Big_Lsi_model();
	  db_conn db =  new db_conn();
	  blsim.conn = db.initdb(blsim.database, blsim.table);
	  blsim.ReadData(blsim.conn);
	  //Remove Stop words
	  //Stemming
	 // lsim.remove_stop_words();
	  
	  blsim.getUniqTerms();
	  
	  //Hack :)
	  for(int i=100;i<blsim.UTerms.size();)
	     {blsim.UTerms.removeElementAt(i);
	     
	     }
	  
	  for(int i=0;i<blsim.UTerms.size();i++)
	  {
		 // System.out.print("Hello");
		  System.out.println("i="+ i+" "+blsim.UTerms.get(i));
		  //System.out.print("Hi");
	  }  
	  
	  blsim.calculateTDM ();
	 // System.out.println("loop " );
		
		for(int j=0;j<blsim.UTerms.size();j++)
		{
			System.out.print("Term["+j+"]="+blsim.UTerms.get(j)+"\t\t ");
			 for(int i=0;i<blsim.data.size();i++)
				 
		  	{
				 try{
			          System.out.print(blsim.TERM_DOCUMENT_MATRIX[j][i]+ " ");
				 }catch(Exception e)
				 {e.printStackTrace();}
		  	}
			 
			 System.out.println(" ");
			
		}//for
	  blsim.TERMS =  new String[blsim.UTerms.size()];
	  blsim.UTerms.toArray(blsim.TERMS);
	  blsim.DOCS =blsim.create_docs();
      
	  /*for(int i=0;i<lsim.TERMS.length;i++)
    	  System.out.println("TERMS["+i+"]=" +lsim.TERMS[i]);*/
	  
	  SvdMatrix matrix= SvdMatrix.svd(blsim.TERM_DOCUMENT_MATRIX,
		                   NUM_FACTORS,
		                   featureInit,
		                   initialLearningRate,
		                   annealingRate,
		                   regularization,
		                   null,
		                   minImprovement,
		                   minEpochs,
		                   maxEpochs);	  
	 
	  //Command line arguments
	  blsim.args = new String[blsim.TERMS.length];
	  blsim.args = blsim.TERMS;
	  blsim.print_various_values(matrix, blsim.args);
	 
	  
	  db.closedb(blsim.conn);
	  
      }//main



}//class
